{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "xf3lVTZYhbzA"
   },
   "source": [
    "# Initial Setups"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2ORFXeezn5Og"
   },
   "source": [
    "## (Google Colab use only)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 42519,
     "status": "ok",
     "timestamp": 1615624277512,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "YFAQ6IgXn8FK",
    "outputId": "50ca7c19-763e-4355-e020-1683a085a9dc"
   },
   "outputs": [],
   "source": [
    "# Use Google Colab\n",
    "use_colab = True\n",
    "\n",
    "# Is this notebook running on Colab?\n",
    "# If so, then google.colab package (github.com/googlecolab/colabtools)\n",
    "# should be available in this environment\n",
    "\n",
    "# Previous version used importlib, but we could do the same thing with\n",
    "# just attempting to import google.colab\n",
    "try:\n",
    "    from google.colab import drive\n",
    "    colab_available = True\n",
    "except:\n",
    "    colab_available = False\n",
    "\n",
    "if use_colab and colab_available:\n",
    "    drive.mount('/content/drive')\n",
    "\n",
    "    # cd to the appropriate working directory under my Google Drive\n",
    "    %cd '/content/drive/My Drive/cs696ds_lexalytics/Prompting Experiments'\n",
    "    \n",
    "    # Install packages specified in requirements\n",
    "    !pip install -r requirements.txt\n",
    "    \n",
    "    # List the directory contents\n",
    "    !ls"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tgzsHF7Zhbzo"
   },
   "source": [
    "## Experiment parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 42516,
     "status": "ok",
     "timestamp": 1615624277515,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "DUpGBmOJhbzs",
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# We will use the following string ID to identify this particular (training) experiments\n",
    "# in directory paths and other settings\n",
    "experiment_id = 'zero_shot_prompt_logit_softmax_atsc_laptops_gpt2_amazon_electronics_multiple_prompts'\n",
    "\n",
    "# Random seed\n",
    "random_seed = 696\n",
    "\n",
    "# path to pretrained MLM model folder or the string \"bert-base-uncased\"\n",
    "lm_model_path = '../progress/lm_further_pretraining_bert_amazon_electronics_bseoh_2021-03-20--13_28_15/results/checkpoint-2360776'\n",
    "\n",
    "# Prompts to be added to the end of each review text\n",
    "# Note: pseudo-labels for each prompt should be given in the order of (positive), (negative), (neutral)\n",
    "sentiment_prompts = {\n",
    "    'i_felt': {\"prompt\": \"I felt the {aspect} was \", \"labels\": [\"good\", \"bad\", \"ok\"]},\n",
    "    'made_me_feel': {\"prompt\": \"The {aspect} made me feel \", \"labels\": [\"good\", \"bad\", \"indifferent\"]},\n",
    "    'the_aspect_is': {\"prompt\": \"The {aspect} is \", \"labels\": [\"good\", \"bad\", \"ok\"]}\n",
    "}\n",
    "\n",
    "# Multiple prompt merging behavior\n",
    "prompts_merge_behavior = 'sum_logits'\n",
    "\n",
    "# Test settings\n",
    "testing_batch_size = 32\n",
    "testing_domain = 'restaurants' # 'laptops', 'restaurants', 'joint'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Experiment ID:\", experiment_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GYZesqTioMvF"
   },
   "source": [
    "## Package imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 50552,
     "status": "ok",
     "timestamp": 1615624285562,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "MlK_-DrWhbzb",
    "outputId": "45b139c8-7f55-4aaa-f223-9f940c64f5f0"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import random\n",
    "import shutil\n",
    "import copy\n",
    "import inspect\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "import transformers\n",
    "import datasets\n",
    "import sklearn.metrics\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sn\n",
    "import tqdm\n",
    "\n",
    "current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n",
    "parent_dir = os.path.dirname(current_dir)\n",
    "sys.path.append(parent_dir)\n",
    "\n",
    "import utils\n",
    "\n",
    "# Random seed settings\n",
    "random.seed(random_seed)\n",
    "np.random.seed(random_seed)\n",
    "\n",
    "# cuBLAS reproducibility\n",
    "# https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility\n",
    "os.environ['CUBLAS_WORKSPACE_CONFIG'] = \":4096:8\"\n",
    "torch.set_deterministic(True)\n",
    "torch.manual_seed(random_seed)\n",
    "\n",
    "# Print version information\n",
    "print(\"Python version: \" + sys.version)\n",
    "print(\"NumPy version: \" + np.__version__)\n",
    "print(\"PyTorch version: \" + torch.__version__)\n",
    "print(\"Transformers version: \" + transformers.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UWuR30eUoTWP"
   },
   "source": [
    "## PyTorch GPU settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 50544,
     "status": "ok",
     "timestamp": 1615624285563,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "PfNlm-ykoSlM",
    "outputId": "e469ee0c-dbd4-4b5a-ebb0-7a5fbd569fb2"
   },
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():    \n",
    "    torch_device = torch.device('cuda')\n",
    "\n",
    "    # Set this to True to make your output immediately reproducible\n",
    "    # Note: https://pytorch.org/docs/stable/notes/randomness.html\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    \n",
    "    # Disable 'benchmark' mode: Set this False if you want to measure running times more fairly\n",
    "    # Note: https://discuss.pytorch.org/t/what-does-torch-backends-cudnn-benchmark-do/5936\n",
    "    torch.backends.cudnn.benchmark = False\n",
    "    \n",
    "    # Faster Host to GPU copies with page-locked memory\n",
    "    use_pin_memory = True\n",
    "    \n",
    "    # Number of compute devices to be used for training\n",
    "    training_device_count = torch.cuda.device_count()\n",
    "\n",
    "    # CUDA libraries version information\n",
    "    print(\"CUDA Version: \" + str(torch.version.cuda))\n",
    "    print(\"cuDNN Version: \" + str(torch.backends.cudnn.version()))\n",
    "    print(\"CUDA Device Name: \" + str(torch.cuda.get_device_name()))\n",
    "    print(\"CUDA Capabilities: \"+ str(torch.cuda.get_device_capability()))\n",
    "    print(\"Number of CUDA devices: \"+ str(training_device_count))\n",
    "    \n",
    "else:\n",
    "    torch_device = torch.device('cpu')\n",
    "    use_pin_memory = False\n",
    "    \n",
    "    # Number of compute devices to be used for training\n",
    "    training_device_count = 1\n",
    "\n",
    "print()\n",
    "print(\"PyTorch device selected:\", torch_device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ayX5VRLfocFk"
   },
   "source": [
    "# Prepare Datasets for Prompt-based Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "U9LAAJP-hbz7"
   },
   "source": [
    "## Load the SemEval dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 53313,
     "status": "ok",
     "timestamp": 1615624288339,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "gpL2uHPUhbz9",
    "outputId": "a90fcc98-39a0-42c7-d32e-6f9115e8eff3"
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    # Load semeval for both domains\n",
    "    laptops_dataset = datasets.load_dataset(\n",
    "        os.path.abspath('../dataset_scripts/semeval2014_task4/semeval2014_task4.py'),\n",
    "        data_files={\n",
    "            'test': '../dataset_files/semeval_2014/Laptops_Test_Gold.xml',\n",
    "            'train': '../dataset_files/semeval_2014/Laptop_Train_v2.xml',\n",
    "        },\n",
    "        cache_dir='../dataset_cache')\n",
    "\n",
    "    restaurants_dataset = datasets.load_dataset(\n",
    "        os.path.abspath('../dataset_scripts/semeval2014_task4/semeval2014_task4.py'),\n",
    "        data_files={\n",
    "            'test': '../dataset_files/semeval_2014/Restaurants_Test_Gold.xml',\n",
    "            'train': '../dataset_files/semeval_2014/Restaurants_Train_v2.xml',\n",
    "        },\n",
    "        cache_dir='../dataset_cache')\n",
    "except:\n",
    "    # Load semeval for both domains\n",
    "    laptops_dataset = datasets.load_dataset(\n",
    "        os.path.abspath('../dataset_scripts/semeval2014_task4/semeval2014_task4.py'),\n",
    "        data_files={\n",
    "            'test': '../dataset_files/semeval_2014/Laptops_Test_Gold.xml',\n",
    "            'train': '../dataset_files/semeval_2014/Laptop_Train_v2.xml',\n",
    "        })\n",
    "\n",
    "    restaurants_dataset = datasets.load_dataset(\n",
    "        os.path.abspath('../dataset_scripts/semeval2014_task4/semeval2014_task4.py'),\n",
    "        data_files={\n",
    "            'test': '../dataset_files/semeval_2014/Restaurants_Test_Gold.xml',\n",
    "            'train': '../dataset_files/semeval_2014/Restaurants_Train_v2.xml',\n",
    "        })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 53310,
     "status": "ok",
     "timestamp": 1615624288340,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "Gi5m8AbPj1iJ"
   },
   "outputs": [],
   "source": [
    "# The dataset chosen for testing\n",
    "if testing_domain == 'laptops':\n",
    "    test_set = laptops_dataset['test']\n",
    "elif testing_domain == 'restaurants':\n",
    "    test_set = restaurants_dataset['test']\n",
    "elif testing_domain == 'joint':\n",
    "    test_set = laptops_dataset['test'] + restaurants_dataset['test']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6TOMmAtIvoZ_"
   },
   "source": [
    "# Zero-shot ATSC with Prompts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3jNAtuv-hbzv"
   },
   "source": [
    "## Load the pretrained LM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 64134,
     "status": "ok",
     "timestamp": 1615624299167,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "En2BmfjVhbzy"
   },
   "outputs": [],
   "source": [
    "# Load pretrained language model\n",
    "lm = transformers.AutoModelForCausalLM.from_pretrained(lm_model_path)\n",
    "try:\n",
    "    tokenizer = transformers.AutoTokenizer.from_pretrained('gpt2', cache_dir='./gpt2_cache')\n",
    "except:\n",
    "    tokenizer = transformers.AutoTokenizer.from_pretrained('gpt2')\n",
    "tokenizer.pad_token = tokenizer.eos_token"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TEIbN5Xthb0o"
   },
   "source": [
    "## Define a new model with non-trainable softmax head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 75031,
     "status": "ok",
     "timestamp": 1615624310071,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "wN3q4Rsopxby",
    "outputId": "5e8ff13e-7a15-4e6d-9c34-094fce229c69"
   },
   "outputs": [],
   "source": [
    "# Encode the pseudo-label words for each sentiment class\n",
    "sentiment_word_ids = []\n",
    "\n",
    "for sp in sentiment_prompts:\n",
    "    sentiment_word_ids.append(\n",
    "        [tokenizer.convert_tokens_to_ids(tokenizer.tokenize(w))[0] for w in sp['labels']])\n",
    "\n",
    "print(sentiment_word_ids)\n",
    "\n",
    "\n",
    "classifier_model = utils.MultiPromptLogitSentimentClassificationHead(\n",
    "    lm=lm,\n",
    "    num_class=3,\n",
    "    num_prompts=len(sentiment_prompts),\n",
    "    pseudo_label_words=sentiment_word_ids,\n",
    "    target_token_id=tokenizer.eos_token_id,\n",
    "    merge_behavior=prompts_merge_behavior)\n",
    "\n",
    "# Freeze the MLM main layer\n",
    "for param in classifier_model.lm.parameters():\n",
    "    param.requires_grad = False\n",
    "\n",
    "classifier_model = classifier_model.to(device=torch_device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1l1H_XIPhb0y"
   },
   "source": [
    "## Evaluation with in-domain test set\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 75030,
     "status": "ok",
     "timestamp": 1615624310073,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "0S80DoYrqApi"
   },
   "outputs": [],
   "source": [
    "def compute_metrics(predictions, labels):\n",
    "    preds = predictions.argmax(-1)\n",
    "\n",
    "    precision, recall, f1, _ = sklearn.metrics.precision_recall_fscore_support(\n",
    "        y_true=labels, y_pred=preds, labels=[0,1,2], average='macro')\n",
    "\n",
    "    acc = sklearn.metrics.accuracy_score(labels, preds)\n",
    "\n",
    "    return {\n",
    "        'accuracy': acc,\n",
    "        'f1': f1,\n",
    "        'precision': precision,\n",
    "        'recall': recall\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 75028,
     "status": "ok",
     "timestamp": 1615624310075,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "9NXoBTs5h2eO"
   },
   "outputs": [],
   "source": [
    "test_dataloader = torch.utils.data.DataLoader(\n",
    "    test_set, batch_size=testing_batch_size, pin_memory=use_pin_memory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 84,
     "referenced_widgets": [
      "7fb8791eebc34b50ad4bbc46d5dd672f",
      "4d37f26bbde14141bbaebb4740551a1a",
      "49b8f6f732824057ba293f03276b16b3",
      "0b6322ca19a5491d913895222b433c0f",
      "718b93fb989d48fda8e7093c8ddd2117",
      "640ccd4f39bd4c5f94d7506ff7883fa4",
      "b71f8e96b97748d98e931eaebfe553c0",
      "3dab522d29284de89d4872f68540ff7e"
     ]
    },
    "executionInfo": {
     "elapsed": 87162,
     "status": "ok",
     "timestamp": 1615624322219,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "LLcc_wZjhb0y",
    "outputId": "70539e45-a5bd-4355-f10d-9654618ffacc"
   },
   "outputs": [],
   "source": [
    "classifier_model.eval()\n",
    "\n",
    "predictions_test = torch.Tensor([])\n",
    "labels_test = torch.Tensor([])\n",
    "\n",
    "for batch_val in tqdm.notebook.tqdm(test_dataloader):\n",
    "\n",
    "    reviews_repeated = []\n",
    "    prompts_populated = []\n",
    "\n",
    "    for prompt in sentiment_prompts:\n",
    "        reviews_repeated = reviews_repeated + batch_val[\"text\"]\n",
    "\n",
    "        for aspect in batch_val[\"aspect\"]:\n",
    "            prompts_populated.append(prompt['prompt'].format(aspect=aspect))\n",
    "\n",
    "    batch_encoded = tokenizer(\n",
    "        reviews_repeated, prompts_populated,\n",
    "        padding='max_length', truncation='only_first', max_length=256,\n",
    "        return_tensors='pt')\n",
    "    \n",
    "    batch_encoded.to(torch_device)\n",
    "\n",
    "    labels = batch_val[\"sentiment\"]\n",
    "\n",
    "    outputs = classifier_model(batch_encoded)\n",
    "\n",
    "    outputs = outputs.to('cpu')\n",
    "\n",
    "    predictions_test = torch.cat([predictions_test, outputs])\n",
    "    labels_test = torch.cat([labels_test, labels])\n",
    "\n",
    "# Compute metrics\n",
    "test_metrics = compute_metrics(predictions_test, labels_test)\n",
    "\n",
    "print(test_metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HjpA_0m1hb08"
   },
   "source": [
    "## Results visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 442
    },
    "executionInfo": {
     "elapsed": 87353,
     "status": "ok",
     "timestamp": 1615624322421,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "w9G9AUeQhb09",
    "outputId": "77a945fa-5896-4acf-d95e-3aa9818f80be"
   },
   "outputs": [],
   "source": [
    "# Calculate metrics and confusion matrix based upon predictions and true labels\n",
    "cm = sklearn.metrics.confusion_matrix(labels_test.detach().numpy(), predictions_test.detach().numpy().argmax(-1))\n",
    "\n",
    "df_cm = pd.DataFrame(\n",
    "    cm,\n",
    "    index=[i for i in [\"positive\", \"negative\", \"neutral\"]],\n",
    "    columns=[i for i in [\"positive\", \"negative\", \"neutral\"]])\n",
    "\n",
    "plt.figure(figsize=(10, 7))\n",
    "\n",
    "ax = sn.heatmap(df_cm, annot=True)\n",
    "\n",
    "ax.set(xlabel='Predicted Label', ylabel='True Label')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "celltoolbar": "Tags",
  "colab": {
   "collapsed_sections": [],
   "name": "prompt_lr_atsc_single_prompt_logit_softmax_the_aspect_bert_amazon_electronics.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0b6322ca19a5491d913895222b433c0f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3dab522d29284de89d4872f68540ff7e",
      "placeholder": "​",
      "style": "IPY_MODEL_b71f8e96b97748d98e931eaebfe553c0",
      "value": " 20/20 [00:12&lt;00:00,  1.64it/s]"
     }
    },
    "3dab522d29284de89d4872f68540ff7e": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "49b8f6f732824057ba293f03276b16b3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_640ccd4f39bd4c5f94d7506ff7883fa4",
      "max": 20,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_718b93fb989d48fda8e7093c8ddd2117",
      "value": 20
     }
    },
    "4d37f26bbde14141bbaebb4740551a1a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "640ccd4f39bd4c5f94d7506ff7883fa4": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "718b93fb989d48fda8e7093c8ddd2117": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "7fb8791eebc34b50ad4bbc46d5dd672f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_49b8f6f732824057ba293f03276b16b3",
       "IPY_MODEL_0b6322ca19a5491d913895222b433c0f"
      ],
      "layout": "IPY_MODEL_4d37f26bbde14141bbaebb4740551a1a"
     }
    },
    "b71f8e96b97748d98e931eaebfe553c0": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
